package ihuotui.sample.webmagic.abstractPage;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.lang.math.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.MonitorableScheduler;
import us.codecraft.webmagic.scheduler.Scheduler;

public abstract class AbstractPageAndSubPageProcessor implements PageProcessor {

    Logger logger = LoggerFactory.getLogger(AbstractPageAndSubPageProcessor.class);

    private AtomicInteger mainPageNo;

    private AtomicBoolean makingNextMainPage;

    private String domain;

    private JedisPool redisPool;

    private Site site;

    private Scheduler scheduler;

    private Task task;

    public AbstractPageAndSubPageProcessor(JedisPool redisPool, String domain, String Referer) {
        this.redisPool = redisPool;
        this.mainPageNo = new AtomicInteger(-1);
        this.domain = domain;
        Set<Integer> acceptStatCode = new HashSet<Integer>();
        acceptStatCode.add(200);
        acceptStatCode.add(503);
        acceptStatCode.add(404);
        this.makingNextMainPage = new AtomicBoolean(false);
        this.site = Site.me().setCycleRetryTimes(5).setRetryTimes(5).setSleepTime(1000)
                .setTimeOut(1 * 60 * 1000)
                .setUserAgent(
                        "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0")
                .addHeader("Accept",
                        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
                .addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
                .setCharset("UTF-8").setAcceptStatCode(acceptStatCode);
        if (StringUtils.isNotBlank(Referer)) {
            this.site.addHeader("Referer", Referer);
        }
        if (StringUtils.isNotBlank(domain)) {
            this.site.setDomain(domain);
        }
    }

    public Scheduler getScheduler() {
        return scheduler;
    }

    public void setScheduler(Scheduler scheduler) {
        this.scheduler = scheduler;
    }

    public Task getTask() {
        return task;
    }

    public void setTask(Task task) {
        this.task = task;
    }

    @SuppressWarnings("static-access")
    @Override
    public void process(Page page) {

        if (mainPageNo.compareAndSet(-1, 2)) {
            Jedis jedis = redisPool.getResource();
            try {
                String rMainPageNo = jedis.get("mainPageNo_" + domain);
                if (StringUtils.isNotBlank(rMainPageNo)) {
                    mainPageNo.set(Integer.valueOf(rMainPageNo));
                } else {
                    jedis.set("mainPageNo_" + domain, String.valueOf(mainPageNo.get()));
                }
            } finally {
                jedis.close();
            }
        }

        MonitorableScheduler queue = (MonitorableScheduler) getScheduler();
        if (queue.getLeftRequestsCount(getTask()) <= 1) {
            manualNextMainPageUrl(page);
        }
        // check haved scan
        if (scaned(page)) {
            return;
        }
        // get next photo url
        if (skipAddTodo(page)) {
            return;
        }

        try {
            Thread.currentThread().sleep(100 * RandomUtils.nextInt(3));
        } catch (InterruptedException e1) {
            e1.printStackTrace();
        }

        if (isMainPage(page)) {
            logger.info("analysis MainPage " + page.getUrl());
            analysisMainPage(page);
            page.addTargetRequests(getSubPageListUrlFromMainPage(page));
        } else {
            logger.info("analysis SubPage " + page.getUrl());
            detailProcess(page);
        }
        addScaned(page);
    }

    public abstract void analysisMainPage(Page page);

    public abstract boolean isMainPage(Page page);

    private boolean scaned(Page page) {
        Jedis jedis = redisPool.getResource();
        try {
            if (jedis.exists("scan_" + domain + "_" + page.getUrl().toString())) {
                return true;
            }
        } finally {
            jedis.close();
        }
        return false;
    }

    private void addScaned(Page page) {
        Jedis jedis = redisPool.getResource();
        try {
            jedis.set("scan_" + domain + "_" + page.getUrl().toString(), "" + page.getStatusCode(),
                    "NX", "EX", 7 * 24 * 60 * 60);
        } finally {
            jedis.close();
        }
    }

    private boolean skipAddTodo(Page page) {
        if (page.getStatusCode() == 503) {
            logger.info("sleep when 503 " + page.getUrl().toString());
            List<String> urls = new LinkedList<String>();
            urls.add(page.getUrl().toString());
            page.addTargetRequests(urls);
            return true;
        } else {
            return false;
        }
    }

    private void manualNextMainPageUrl(Page page) {
        if (makingNextMainPage.compareAndSet(false, true)) {
            makingNextMainPage.set(false);
            if (mainPageNo.intValue() > maxMainPage()) {
                return;
            } else {
                List<String> nextMainPage = new LinkedList<String>();
                String makeNext = null;
                Integer mainPageNo = this.mainPageNo.getAndIncrement();
                makeNext = makeNextMainPage(mainPageNo);
                if (StringUtils.isBlank(makeNext)) {
                    return;
                }
                nextMainPage.add(makeNext);
                page.addTargetRequests(nextMainPage);
                logger.info("makeNextMainPage " + makeNext);

                Jedis jedis = redisPool.getResource();
                try {
                    jedis.set("mainPageNo_" + domain, String.valueOf(mainPageNo));
                } finally {
                    jedis.close();
                }
            }
        }
    }

    public abstract int maxMainPage();

    public abstract String makeNextMainPage(Integer mainPageNo);

    private void detailProcess(Page page) {
        boolean isMakeNextSubPage = false;
        analysisSubPage(page, isMakeNextSubPage);
        if (isMakeNextSubPage) {
            List<String> nextSubPage = new ArrayList<String>(1);
            nextSubPage.add(makeSubPageNext(page));
            if (StringUtils.isNotBlank(nextSubPage.get(0))) {
                page.addTargetRequests(nextSubPage);
                logger.info("makeNextDetailPage " + nextSubPage.get(0));
            }
        }
    }

    public abstract void analysisSubPage(Page page, boolean isMakeNextSubPage);

    public abstract String makeSubPageNext(Page page);

    public abstract List<String> getSubPageListUrlFromMainPage(Page page);

    @Override
    public Site getSite() {
        return site;
    }
}
